import requests
from lxml import etree
from urllib.parse import quote
import json


def parse(url):
    lua = '''function main(splash, args)
      splash.private_mode_enabled = false
      assert(splash:go("''' + url + '''"))
      assert(splash:wait(0.5))
      splash:set_user_agent("Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36")
      return {
        html = splash:html(),
      }
    end'''
    response = requests.get("http://localhost:7000/execute?lua_source=" + quote(lua))
    data = json.loads(response.text)["html"]
    html = etree.HTML(data)
    data = json.loads(html.xpath('/html/body/pre/text()')[0])
    nodes = data['data']['item']
    List = []
    for node in nodes:
        List.append({k: node[k] for k in
                     ['address', 'address', 'city_id', 'city_name', 'county_id', 'county_name', 'dual_class_name',
                      'f211', 'f985', 'level_name', 'name', 'nature_name', 'province_id', 'province_name',
                      'school_id', 'type_name']})
    return List


if __name__ == '__main__':
    url = 'https://api.eol.cn/gkcx/api/?page=1&request_type=1&size=30&uri=apigkcx/api/school/hotlists'
    for index in range(1, 96):
        List = parse(f'https://api.eol.cn/gkcx/api/?page={index}&request_type=1&size=30&uri=apigkcx/api/school/hotlists')
        with open(f'./school_id/{index}.json', 'w', encoding='utf-8') as f:
            json.dump(List, f, ensure_ascii=False, indent=4)
            print(f'保存了{index}个文件')

